package com.doit.demo.day02.sources;

import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;

import java.util.Properties;

/**
 * @DATE 2022/2/14/11:33
 * @Author MDK
 * @Version 2021.2.2
 *
 * Flink从kafka中读取数据
 *  1.KafkaSource是多并行的Source吗?
 *  2.Kafka对应的subtask数量如果与topic分区数不一致怎么样
 *      当subtask数量>topic分区时,会有subtask空跑
 *      当subtask数量<topic分区时,一个subtask会消费多个分区的数据
 *  3.同一个组消费的数据如果重启,会不会重新消费?(消费者有没有记录偏移量,偏移量记录到哪里了)
 *      偏移量记录在kafka特殊的topic中,状态
 **/
public class KafkaSourceDemo {
    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        configuration.setInteger("rest.port",8081);
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        Properties properties = new Properties();
        properties.setProperty("boostrap.servers","linux01:9092,linux02:9092,linux03:9092");
        properties.setProperty("group.id","test");


        //没有偏移量从头开始读取,有偏移量接着偏移量读取
        properties.setProperty("auto.offset.reset","earliest");
        //不将偏移量写入kafka的特殊性topic中
        properties.setProperty("enable.auto.commit","false");

        
        //调用addSource添加Flink的Connector
        FlinkKafkaConsumer<String> flinkKafkaConsumer = new FlinkKafkaConsumer<>(
                "test",
                new SimpleStringSchema(),
                properties
        );
        flinkKafkaConsumer.setStartFromEarliest();
        DataStreamSource<String> lines = env.addSource(flinkKafkaConsumer);

        lines.print();

        env.execute();
    }
}
